import requests
from queue import Queue
from bs4 import BeautifulSoup
import os
import re
from concurrent.futures import ThreadPoolExecutor
import threading
class Bole:
    parent_dir='downloads/'   #文件夹名字
    def __init__(self):
        self.list_url_queue=Queue(maxsize=1000)
        self.list_url_queue.put('http://blog.jobbole.com/all-posts/')
        self.detail_url=Queue(maxsize=10000)
        self.detail_urlpool=ThreadPoolExecutor(max_workers=1)
        self.get_contentpool=ThreadPoolExecutor(max_workers=1)
        self.all_list=[]
        self.num=0
    def get_list_url(self,bs):
        """
        或得翻页url
        :return:
        """

        list_a=bs.select('.page-numbers')
        for i in list_a:
            try:
                if i['href'] not in self.all_list:
                    self.all_list.append(i['href'])
                    self.list_url_queue.put(i['href'])
                    print('目前列表大小：',self.list_url_queue.qsize())
            except Exception :
                continue
    def get_detail_url(self):
        """
        获得详情页url
        :return:
        """
        url=self.list_url_queue.get()
        re=requests.get(url)
        bs=BeautifulSoup(re.text)
        self.get_list_url(bs)
        try:
            detail_a=bs.select('.archive-title')
        except Exception as e:
            pass

        for i in detail_a:
            href=i['href']
            self.detail_url.put(href)

    def get_content_save(self):
        """
        获得详情页并保存
        :return:
        """
        url=self.detail_url.get()
        re=requests.get(url)
        filename=self.getname(url)
        self.num+=1
        print(self.num)
        self.save(filename,re.text)
    def save(self,filename,content):
        path=os.path.join(self.parent_dir,filename)
        path=path+'.html'
        with open(path,'w+',encoding='utf8') as f:
            f.write(content)
    def getname(self,url):
        return re.findall('\d+',url)[0]
    def loop_get_detail_url(self):
        """线程池执行获得详情页url"""
        while True:
            self.detail_urlpool.submit(self.get_detail_url)
    def loop_get_content(self):
        """
        线程池执行获得内容
        :return:
        """
        while True:
            self.get_contentpool.submit(self.get_content_save)
    def main(self):
        """
        整合上面的线程池
        :return:
        """
        threading.Thread(target=self.loop_get_detail_url).start()
        threading.Thread(target=self.loop_get_content).start()
if __name__ == '__main__':
    b = Bole()
    b.main()




